package phases

import (
	"crypto/sha256"
	"fmt"
	"k8s.io/client-go/dynamic"
	"strings"

	"github.com/pkg/errors"
	agentv1beta1 "gopkg.openfuyao.cn/bkeagent/api/v1beta1"
	downloadplugin "gopkg.openfuyao.cn/bkeagent/pkg/job/builtin/downloader"
	mfplugin "gopkg.openfuyao.cn/bkeagent/pkg/job/builtin/kubeadm/manifests"
	"gopkg.openfuyao.cn/bkeagent/utils/pkiutil"
	"gopkg.openfuyao.cn/bkecommon"
	bkeaddon "gopkg.openfuyao.cn/bkecommon/cluster/addon"
	confv1beta1 "gopkg.openfuyao.cn/bkecommon/cluster/api/v1beta1"
	bkeinit "gopkg.openfuyao.cn/bkecommon/cluster/initialize"
	bkenode "gopkg.openfuyao.cn/bkecommon/cluster/node"
	bkesource "gopkg.openfuyao.cn/bkecommon/source"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/client-go/kubernetes"
	"sigs.k8s.io/cluster-api/util/annotations"
	ctrl "sigs.k8s.io/controller-runtime"

	bkev1beta1 "gopkg.openfuyao.cn/cluster-api-provider-bke/api/v1beta1"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/pkg/certs"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/pkg/command"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/pkg/kube"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/pkg/mergecluster"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/pkg/phaseframe"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/pkg/phaseframe/phaseutil"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/utils"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/utils/annotation"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/utils/clusterutil"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/utils/condition"
	"gopkg.openfuyao.cn/cluster-api-provider-bke/utils/constant"
	labelhelper "gopkg.openfuyao.cn/cluster-api-provider-bke/utils/label"
)

const (
	EnsureAddonDeployName confv1beta1.BKEClusterPhase = "EnsureAddonDeploy"

	etcdCertsScretName = "etcd-backup-secrets"
)

type EnsureAddonDeploy struct {
	phaseframe.BasePhase
	addons              []*bkeaddon.AddonTransfer
	targetClusterClient kube.RemoteKubeClient
	remoteClient        *kubernetes.Clientset
	remoteDynamicClient dynamic.Interface
	addonRecorders      []*kube.AddonRecorder
}

func NewEnsureAddonDeploy(ctx *phaseframe.PhaseContext) phaseframe.Phase {
	base := phaseframe.NewBasePhase(ctx, EnsureAddonDeployName)
	phase := &EnsureAddonDeploy{BasePhase: base}
	phase.RegisterPostHooks(phase.saveAddonManifestsPostHook)
	return phase
}

func (e *EnsureAddonDeploy) Execute() (_ ctrl.Result, err error) {
	targetClusterClient, err := kube.NewRemoteClientByBKECluster(e.Ctx.Context, e.Ctx.Client, e.Ctx.BKECluster)
	if err != nil {
		return ctrl.Result{}, err
	}
	e.targetClusterClient = targetClusterClient
	e.targetClusterClient.SetLogger(e.Ctx.Log.NormalLogger)
	e.targetClusterClient.SetBKELogger(e.Ctx.Log)
	e.remoteClient, e.remoteDynamicClient = targetClusterClient.KubeClient()

	if err = e.reconcileAddon(); err != nil {
		return ctrl.Result{}, err
	}
	return ctrl.Result{}, nil
}

func (e *EnsureAddonDeploy) NeedExecute(old *bkev1beta1.BKECluster, new *bkev1beta1.BKECluster) (needExecute bool) {
	if !e.BasePhase.DefaultNeedExecute(old, new) {
		return false
	}

	if e.Ctx.Cluster != nil && !phaseutil.AllowDeployAddon(new, e.Ctx.Cluster) {
		return false
	}

	if new.Status.NodesStatus == nil && new.Spec.ClusterConfig != nil {
		return new.Spec.ClusterConfig.Addons != nil
	}

	_, ok := bkeaddon.CompareBKEConfigAddon(new.Status.AddonStatus, new.Spec.ClusterConfig.Addons)
	if !ok {
		return false
	}
	e.SetStatus(bkev1beta1.PhaseWaiting)
	return true
}

func (e *EnsureAddonDeploy) reconcileAddon() error {
	_, c, bkeCluster, _, log := e.Ctx.Untie()
	addonsT, ok := bkeaddon.CompareBKEConfigAddon(bkeCluster.Status.AddonStatus, bkeCluster.Spec.ClusterConfig.Addons)
	if !ok {
		return nil
	}

	log.Info(constant.AddonDeployingReason, "start to reconcile addon for target cluster %q", bkeCluster.Name)
	log.Info(constant.AddonDeployingReason, "A total of %d Addons need to be reconcile", len(addonsT))

	condition.ConditionMark(bkeCluster, bkev1beta1.ClusterAddonCondition, confv1beta1.ConditionFalse, "", "addon deloying")

	var errs []error
	for _, addonT := range addonsT {
		// 对于暂停的BKECluster，不执行
		if bkeCluster.Spec.Pause || annotations.HasPaused(bkeCluster) {
			log.Info(constant.AddonDeployedReason, "BKECluster deploy paused, stop deploy addon")
			break
		}

		if addonT.Operate == bkeaddon.CreateAddon {
			if err := e.addonBeforeCreateCustomOperate(addonT.Addon); err != nil {
				log.Error(constant.AddonDeployFailedReason, "addon %q before create custom operate failed: %s", addonT.Addon.Name, err.Error())
				errs = append(errs, err)
				break
			}
		}

		log.Info(constant.AddonDeployingReason, "start to %s addon %q", addonT.Operate, addonT.Addon.Name)

		addonRecorder := kube.NewAddonRecorder(addonT)
		operateErr := e.targetClusterClient.InstallAddon(bkeCluster, addonT, addonRecorder)

		// 操作单个addon可能会花费许多时间这期间使用者看可能会修改bc，需要使用最新的bc来操作
		newestBkeCluster, err := e.Ctx.GetNewestBKECluster()
		if err != nil {
			return err
		}

		if operateErr != nil {
			errs = append(errs, errors.Wrap(operateErr, addonT.Addon.Name))
			errInfo := fmt.Sprintf("%s %q failed: %s", addonT.Operate, addonT.Addon.Name, operateErr.Error())
			condition.AddonConditionMark(newestBkeCluster, confv1beta1.ClusterConditionType(addonT.Addon.Name), confv1beta1.ConditionFalse, "ReconcileAddonError", errInfo, addonT.Addon.Name)

			if addonT.Addon.Block {
				log.Error(constant.AddonDeployFailedReason, "%s %q failed: %s", addonT.Operate, addonT.Addon.Name, operateErr.Error())
				return operateErr
			} else {
				log.Warn(constant.AddonDeployFailedReason, "%s %q failed (ignore): %s", addonT.Operate, addonT.Addon.Name, operateErr.Error())
				continue
			}
		} else {
			successInfo := fmt.Sprintf("%s %s/%s success", addonT.Operate, addonT.Addon.Name, addonT.Addon.Version)
			condition.AddonConditionMark(newestBkeCluster, confv1beta1.ClusterConditionType(addonT.Addon.Name), confv1beta1.ConditionTrue, "ReconcileAddonSuccess", successInfo, addonT.Addon.Name)
			log.Info(constant.AddonDeploySucceededReason, successInfo)
		}

		switch addonT.Operate {
		case bkeaddon.RemoveAddon:
			condition.RemoveCondition(confv1beta1.ClusterConditionType(addonT.Addon.Name), bkeCluster)
			// remove addon from status
			for i, addon := range newestBkeCluster.Status.AddonStatus {
				if addon.Name == addonT.Addon.Name {
					newestBkeCluster.Status.AddonStatus = append(newestBkeCluster.Status.AddonStatus[:i], newestBkeCluster.Status.AddonStatus[i+1:]...)
					break
				}
			}
		case bkeaddon.UpdateAddon:
			fallthrough
		case bkeaddon.UpgradeAddon:
			// update Status
			for i, addon := range newestBkeCluster.Status.AddonStatus {
				if addon.Name == addonT.Addon.Name {
					newestBkeCluster.Status.AddonStatus[i] = *addonT.Addon
					break
				}
			}
		case bkeaddon.CreateAddon:
			// update Status
			newestBkeCluster.Status.AddonStatus = append(newestBkeCluster.Status.AddonStatus, *addonT.Addon)
			e.addonAfterCreateCustomOperate(addonT.Addon, newestBkeCluster)
		}

		// 不删除都记录
		if addonT.Operate != bkeaddon.RemoveAddon {
			e.addonRecorders = append(e.addonRecorders, addonRecorder)
		}

		if err = mergecluster.SyncStatusUntilComplete(c, newestBkeCluster); err != nil {
			log.Error(constant.AddonDeployFailedReason, "update bkecluster %q Status failed: %s", bkeCluster.Name, err.Error())
			return err
		}
		err = e.Ctx.RefreshCtxBKECluster()
		if err != nil {
			return err
		}
		bkeCluster = newestBkeCluster
	}
	log.Info(constant.AddonDeployedReason, "%d Addons were reconciled, %d succeeded, %d failed", len(addonsT), len(addonsT)-len(errs), len(errs))
	if len(errs) > 0 {
		log.Warn(constant.AddonDeployedReason, "reconcile addons failed: %s", errs)
		return kerrors.NewAggregate(errs)
	}
	condition.ConditionMark(bkeCluster, bkev1beta1.ClusterAddonCondition, confv1beta1.ConditionTrue, "", "")

	return nil
}

func (e *EnsureAddonDeploy) addonBeforeCreateCustomOperate(addon *confv1beta1.Product) (err error) {
	switch addon.Name {
	case "etcdbackup":
		if err := e.createEtcdBackupDir(addon.Param["backupDir"]); err != nil {
			e.Ctx.Log.Warn("DeployAddonError", "create etcdbackup dir failed: %s", err.Error())
			return err
		}
		if err := e.createEtcdCertSecret(); err != nil {
			e.Ctx.Log.Warn("DeployAddonError", "create etcd cert secret for addon etcdbackup failed: %s", err.Error())
			return err
		}
	case "beyondELB":
		vip, lbNodes := clusterutil.GetIngressConfig(e.Ctx.BKECluster.Spec.ClusterConfig.Addons)
		if vip == "" && len(lbNodes) == 0 {
			e.Ctx.Log.Warn("DeployAddonWarn", "(bke ignore)create beyondELB failed: vip or lbNodes is empty")
		} else {
			if err := e.createBeyondELBVIP(vip, lbNodes); err != nil {
				e.Ctx.Log.Warn("DeployAddonError", "create beyondELB VIP failed: %s", err.Error())
				return err
			}
			if err := e.labelNodesForELB(lbNodes); err != nil {
				e.Ctx.Log.Warn("DeployAddonError", "label nodes for beyondELB failed: %s", err.Error())
				return err
			}
		}
	case "cluster-api":
		if err := e.createClusterAPILocalkubeconfigSecret(); err != nil {
			e.Ctx.Log.Warn("DeployAddonError", "create cluster-api local kubeconfig secret failed: %s", err.Error())
			return err
		}
		if err := e.switchBKEAgentToBKECluster(); err != nil {
			e.Ctx.Log.Warn("DeployAddonError", "switch bkeagent to bkecluster failed: %s", err.Error())
			return err
		}
		if err := e.createClusterAPIBkeconfigCm(); err != nil {
			e.Ctx.Log.Warn("DeployAddonError", "create cluster-api bke config configmap failed: %s", err.Error())
			return err
		}
	case constant.OpenFuyaoSystemController:
		if err := e.addControlPlaneLabels(); err != nil {
			e.Ctx.Log.Warn("AddLabelsError", "add control plane labels failed: %s", err.Error())
		}
	case "gpu-manager":
		if err = e.reCreateKubeSchedulerStaticPodYaml(); err != nil {
			e.Ctx.Log.Error("DeployAddonError", "recreate kube-scheduler static pod yaml failed: %s", err.Error())
			return err
		}
	// 在环境初始化时额外处理
	//case "calico":
	//	if err = e.downloadCalicoCtl(addon.Version); err != nil {
	//		e.Ctx.Log.Warn("DeployAddonError", "config calicoctl failed: %s", err.Error())
	//		return nil
	//	}
	default:
	}
	return
}

func (e *EnsureAddonDeploy) addonAfterCreateCustomOperate(addon *confv1beta1.Product, bkeCluster *bkev1beta1.BKECluster) {
	e.Ctx.Log.Info("CustomOperate", fmt.Sprintf("addon %q after create custom operate", addon.Name))
	switch addon.Name {
	case constant.OpenFuyaoSystemController:
		// 创建默认的用户名和密码
		cfg := phaseutil.UserInfoConfig{
			PasswdLength:  12,
			SaltLength:    16,
			Iterations:    100000,
			KeyLength:     64,
			EncryptMethod: sha256.New,
		}

		username, passwd, err := phaseutil.GenerateDefaultUserInfo(e.remoteDynamicClient, cfg)
		if err != nil {
			e.Ctx.Log.Error("NotCreateDefaultUser", err.Error())
			return
		}
		if len(passwd) == 0 {
			e.Ctx.Log.Info("re-install", "re-install openFuyao-system")
			return
		}

		msg := `

	The website of the openFuyao is as follows:
	
		https://%s:%s

	You can login to the openFuyao using the following username and password:

		username: %s
		password: %s

`
		e.Ctx.Log.Info("openFuyaoSystemReady", fmt.Sprintf(msg, e.Ctx.BKECluster.Spec.ControlPlaneEndpoint.Host,
			constant.OpenFuyaoSystemPort, username, passwd))
	default:
		e.Ctx.Log.Info("CustomOperate", "no custom operate")
	}
}

func (e *EnsureAddonDeploy) addControlPlaneLabels() error {
	ctx, _, bkeCluster, _, log := e.Ctx.Untie()

	remoteNodes, err := e.targetClusterClient.ListNodes(nil)
	if err != nil {
		e.Ctx.Log.Error("GetNodeLabelFailed", "failed to list nodes before deploy openfuyao-system, err: %v",
			err)
		return fmt.Errorf("failed to list nodes before deploy openfuyao-system: %v", err)
	}

	bkeNodes := bkenode.Nodes(bkeCluster.Spec.ClusterConfig.Nodes).Master()
	for _, node := range remoteNodes.Items {
		for _, bkeNode := range bkeNodes {
			if node.GetName() != bkeNode.Hostname {
				continue
			}
			log.Info("BeforeCreateCustomOperate", "label node %s as control-plane", node.GetName())
			labelhelper.SetMasterRoleLabel(&node)
			_, err := e.remoteClient.CoreV1().Nodes().Update(ctx, &node, metav1.UpdateOptions{})
			if err != nil {
				return errors.Errorf("failed to set control-plane label to node %s: %s", node.Name, err.Error())
			}
			log.Info("BeforeCreateCustomOperate", "set control-plane label to node %s success", node.Name)
		}
	}
	return nil
}

func (e *EnsureAddonDeploy) createEtcdBackupDir(dir string) error {
	ctx, c, bkeCluster, scheme, log := e.Ctx.Untie()

	bkeNodes := bkenode.Nodes(bkeCluster.Spec.ClusterConfig.Nodes)

	createDirCommand := command.Custom{
		BaseCommand: command.BaseCommand{
			Ctx:             ctx,
			NameSpace:       bkeCluster.Namespace,
			Client:          c,
			Scheme:          scheme,
			OwnerObj:        bkeCluster,
			ClusterName:     bkeCluster.Name,
			Unique:          true,
			RemoveAfterWait: true,
		},
		Nodes:        bkeNodes.Etcd(),
		CommandName:  "create-etcd-backup-dir",
		CommandLabel: command.BKEClusterLabel,
	}

	commandSpec := command.GenerateDefaultCommandSpec()
	commandSpec.Commands = []agentv1beta1.ExecCommand{
		{
			ID: "mkdir",
			Command: []string{
				fmt.Sprintf("mkdir -p %s", dir),
			},
			Type:          agentv1beta1.CommandShell,
			BackoffIgnore: false,
		},
	}
	createDirCommand.CommandSpec = commandSpec

	if err := createDirCommand.New(); err != nil {
		return errors.Errorf("create create-etcd-backup-dir command failed: %s", err.Error())
	}

	err, successNodes, failedNodes := createDirCommand.Wait()
	if err != nil {
		return errors.Errorf("create create-etcd-backup-dir command failed: %s", err.Error())
	}
	if len(failedNodes) > 0 {
		errInfo := "create-etcd-backup-dir command run failed"
		commandErrs, err := phaseutil.LogCommandFailed(*createDirCommand.Command, failedNodes, log, "create beyondELB VIP failed")
		phaseutil.MarkNodeStatusByCommandErrs(bkeCluster, commandErrs)
		return errors.Errorf("%s, createDirCommand run failed in flow nodes: %v, err: %v", errInfo, strings.Join(failedNodes, ","), err)
	}
	log.Info("BeforeCreateCustomOperate", "create etcd backup dir to fellow nodes success: %v", successNodes)
	return nil
}

func (e *EnsureAddonDeploy) createEtcdCertSecret() error {
	ctx, c, bkeCluster, _, log := e.Ctx.Untie()
	// 获取secret
	_, err := e.remoteClient.CoreV1().Secrets("kube-system").Get(ctx, etcdCertsScretName, metav1.GetOptions{})
	if err != nil {
		if !apierrors.IsNotFound(err) {
			return err
		}

		log.Debug("create etcd cert secret for addon etcdbackup")

		certGetter := certs.NewBKEKubernetesCertGetter(ctx, c, bkeCluster)

		etcdCaCertContent, err := certGetter.GetCertContent(pkiutil.BKECertEtcdCA())
		if err != nil {
			return err
		}
		etcdClientCertContent, err := certGetter.GetCertContent(pkiutil.BKECertEtcdAPIClient())
		if err != nil {
			return err
		}

		etcdBackupSecret := &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      etcdCertsScretName,
				Namespace: "kube-system",
			},
			StringData: map[string]string{
				"etcd-ca":   etcdCaCertContent.Cert,
				"etcd-cert": etcdClientCertContent.Cert,
				"etcd-key":  etcdClientCertContent.Key,
			},
		}
		_, err = e.remoteClient.CoreV1().Secrets(etcdBackupSecret.Namespace).Create(ctx, etcdBackupSecret, metav1.CreateOptions{})
		if err != nil {
			return err
		}
	}

	log.Info("BeforeCreateCustomOperate", "create etcd cert secret for addon etcdbackup success")
	return nil
}

func (e *EnsureAddonDeploy) createBeyondELBVIP(vip string, lbNodes []string) error {
	if vip == "" {
		return nil
	}

	ctx, c, bkeCluster, scheme, log := e.Ctx.Untie()

	cfg := bkeinit.BkeConfig(*bkeCluster.Spec.ClusterConfig)

	ingressNodes := phaseutil.ConvertELBNodesToBKENodes(lbNodes, bkeCluster.Spec.ClusterConfig.Nodes)
	if len(ingressNodes) == 0 {
		return fmt.Errorf("no ingress node set for beyondELB")
	}

	log.Info("BeforeCreateCustomOperate", "create beyondELB VIP: %s to follow node(s): %v, please wait", vip, lbNodes)

	VIPCommand := command.HA{
		BaseCommand: command.BaseCommand{
			Ctx:         ctx,
			NameSpace:   bkeCluster.Namespace,
			Client:      c,
			Scheme:      scheme,
			OwnerObj:    bkeCluster,
			ClusterName: bkeCluster.Name,
			Unique:      true,
		},
		IngressNodes:    ingressNodes,
		IngressVIP:      vip,
		ImageRepo:       cfg.ImageRepo(),
		ManifestsDir:    cfg.Cluster.Kubelet.ManifestsDir,
		VirtualRouterId: cfg.CustomExtra["ingressVirtualRouterId"],
		WaitVIP:         true,
	}

	if err := VIPCommand.New(); err != nil {
		errInfo := "failed to create beyondELB VIP command"
		return errors.Wrap(err, errInfo)
	}

	err, _, failedNodes := VIPCommand.Wait()
	if err != nil {
		errInfo := "failed to create beyondELB VIP"
		return errors.Wrap(err, errInfo)
	}
	if len(failedNodes) > 0 {
		errInfo := "failed to create beyondELB VIP"
		commandErrs, err := phaseutil.LogCommandFailed(*VIPCommand.Command, failedNodes, log, "create beyondELB VIP failed")
		phaseutil.MarkNodeStatusByCommandErrs(bkeCluster, commandErrs)
		return errors.Errorf("%s, VIPCommand run failed in flow nodes: %v, err: %v", errInfo, strings.Join(failedNodes, ","), err)
	}

	log.Info("BeforeCreateCustomOperate", "create beyondELB VIP success")
	return nil
}

func (e *EnsureAddonDeploy) labelNodesForELB(lbNodes []string) error {
	ctx, _, bkeCluster, _, log := e.Ctx.Untie()

	ingressNodes := phaseutil.ConvertELBNodesToBKENodes(lbNodes, bkeCluster.Spec.ClusterConfig.Nodes)

	// set loadblance label to remote nodes
	remoteNodes, err := e.targetClusterClient.ListNodes(nil)
	if err != nil {
		return errors.Errorf("failed to list remote nodes: %s", err.Error())
	}

	labelNodes := []corev1.Node{}
	for _, elbNode := range ingressNodes {
		found := false
		for _, node := range remoteNodes.Items {
			for _, addr := range node.Status.Addresses {
				if addr.Type == corev1.NodeInternalIP {
					if addr.Address == elbNode.IP {
						found = true
						break
					}
				}
				if addr.Type == corev1.NodeHostName {
					if addr.Address == elbNode.Hostname {
						found = true
						break
					}
				}
			}
			if found {
				labelNodes = append(labelNodes, node)
				break
			}
		}
	}

	for _, node := range labelNodes {
		labelhelper.SetLabel(&node, labelhelper.BeyondELBLabelKey, labelhelper.BeyondELBLabelValue)
		_, err := e.remoteClient.CoreV1().Nodes().Update(ctx, &node, metav1.UpdateOptions{})
		if err != nil {
			return errors.Errorf("failed to set beyondELB label to node %s: %s", node.Name, err.Error())
		}
		log.Info("BeforeCreateCustomOperate", "set beyondELB label to node %s success", node.Name)

	}
	return nil
}

func (e *EnsureAddonDeploy) createClusterAPILocalkubeconfigSecret() error {
	ctx, c, bkeCluster, _, log := e.Ctx.Untie()
	// 获取secret
	_, err := e.remoteClient.CoreV1().Secrets("kube-system").Get(ctx, constant.LocalKubeConfigName, metav1.GetOptions{})
	if err != nil {
		if !apierrors.IsNotFound(err) {
			return err
		}
		log.Debug("create localkubeconfig secret for cluster-api")
		certGetter := certs.NewBKEKubernetesCertGetter(ctx, c, bkeCluster)
		kubeconfigContent, err := certGetter.GetTargetClusterKubeconfig()
		if err != nil {
			return err
		}

		localKubeconfigSecret := &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      constant.LocalKubeConfigName,
				Namespace: "kube-system",
			},
			StringData: map[string]string{
				"config": kubeconfigContent,
			},
		}
		_, err = e.remoteClient.CoreV1().Secrets(localKubeconfigSecret.Namespace).Create(ctx, localKubeconfigSecret, metav1.CreateOptions{})
		if err != nil {
			return err
		}
	}

	log.Info("BeforeCreateCustomOperate", "create localkubeconfig secret for cluster-api success")
	return nil
}

func (e *EnsureAddonDeploy) createClusterAPIBkeconfigCm() error {
	ctx, c, bkeCluster, _, log := e.Ctx.Untie()
	config, err := phaseutil.GetRemoteBKEConfigCM(ctx, e.remoteClient)
	if err != nil {
		log.Error(constant.InternalErrorReason, "failed to get BKECluster %q remote cluster bke-config cm, err: %v", utils.ClientObjNS(bkeCluster), err)
		return err
	}
	if config == nil {
		if err = phaseutil.MigrateBKEConfigCM(ctx, c, e.remoteClient); err != nil {
			log.Error(constant.InternalErrorReason, "failed to migrate BKECluster %q bke-config cm to remote cluster, err：%v", utils.ClientObjNS(bkeCluster), err)
			return err
		}
	}
	log.Info("BeforeCreateCustomOperate", "create cluster-api bke-config cm for cluster-api success")
	return nil
}

func (e *EnsureAddonDeploy) switchBKEAgentToBKECluster() error {
	ctx, c, bkeCluster, scheme, log := e.Ctx.Untie()

	switchCommand := command.Switch{
		BaseCommand: command.BaseCommand{
			Ctx:         ctx,
			Client:      c,
			NameSpace:   bkeCluster.Namespace,
			Scheme:      scheme,
			OwnerObj:    bkeCluster,
			ClusterName: bkeCluster.Name,
			Unique:      true,
		},
		Nodes:       bkenode.Nodes(bkeCluster.Spec.ClusterConfig.Nodes),
		ClusterName: bkeCluster.Name,
	}
	if err := switchCommand.New(); err != nil {
		log.Finish(constant.SwitchClusterFailedReason, "BKEAgent switch BKECluster %q failed: %s", utils.ClientObjNS(bkeCluster), err.Error())
		return err
	}
	condition.ConditionMark(bkeCluster, bkev1beta1.SwitchBKEAgentCondition, confv1beta1.ConditionTrue, constant.SwitchClusterSuccessReason, fmt.Sprintf("switch BKEAgent to BKECluster %q success", utils.ClientObjNS(bkeCluster)))
	annotation.SetAnnotation(bkeCluster, bkecommon.BKEAgentListenerAnnotationKey, bkecommon.BKEAgentListenerBkecluster)
	log.Info("BeforeCreateCustomOperate", "BKEAgent switch BKECluster %q success", utils.ClientObjNS(bkeCluster))
	return mergecluster.SyncStatusUntilComplete(c, bkeCluster)
}

func (e *EnsureAddonDeploy) reCreateKubeSchedulerStaticPodYaml() error {
	ctx, c, bkeCluster, scheme, log := e.Ctx.Untie()

	bkeNodes := bkenode.Nodes(bkeCluster.Spec.ClusterConfig.Nodes)

	reCreateCommand := command.Custom{
		BaseCommand: command.BaseCommand{
			Ctx:             ctx,
			NameSpace:       bkeCluster.Namespace,
			Client:          c,
			Scheme:          scheme,
			OwnerObj:        bkeCluster,
			ClusterName:     bkeCluster.Name,
			Unique:          true,
			RemoveAfterWait: true,
		},
		Nodes:        bkeNodes.Master(),
		CommandName:  "recreate-kube-scheduler-static-pod-yaml",
		CommandLabel: command.BKEClusterLabel,
	}

	commandSpec := command.GenerateDefaultCommandSpec()
	commandSpec.Commands = []agentv1beta1.ExecCommand{
		{
			ID: "recreate-kube-scheduler-static-pod-yaml",
			Command: []string{
				mfplugin.Name,
				fmt.Sprintf("bkeConfig=%s:%s", bkeCluster.Namespace, bkeCluster.Name),
				"gpuEnable=true",
				"scope=kube-scheduler",
				fmt.Sprintf("manifestDir=%s", bkeCluster.Spec.ClusterConfig.Cluster.Kubelet.ManifestsDir),
			},
			Type:          agentv1beta1.CommandBuiltIn,
			BackoffIgnore: false,
		},
	}
	reCreateCommand.CommandSpec = commandSpec

	if err := reCreateCommand.New(); err != nil {
		return errors.Errorf("create recreate kube-scheduler static pod yaml command failed: %s", err.Error())
	}

	err, successNodes, failedNodes := reCreateCommand.Wait()
	if err != nil {
		return errors.Errorf("wait recreate kube-scheduler static pod yaml command failed: %s", err.Error())
	}
	if len(failedNodes) > 0 {
		errInfo := "recreate-kube-scheduler-static-pod-yaml command run failed"
		commandErrs, err := phaseutil.LogCommandFailed(*reCreateCommand.Command, failedNodes, log, "recreate kube-scheduler static pod yaml failed")
		phaseutil.MarkNodeStatusByCommandErrs(bkeCluster, commandErrs)
		return errors.Errorf("%s, recreate kube-scheduler static pod yaml command run failed in flow nodes: %v, err: %v", errInfo, strings.Join(failedNodes, ","), err)
	}
	log.Info("BeforeCreateCustomOperate", "recreate kube-scheduler static pod yaml command run success in flow nodes: %v", strings.Join(successNodes, ","))
	return nil
}

func (e *EnsureAddonDeploy) downloadCalicoCtl(version string) error {
	ctx, c, bkeCluster, scheme, log := e.Ctx.Untie()
	cfg := bkeinit.BkeConfig(*bkeCluster.Spec.ClusterConfig)
	baseUrl := bkesource.GetCustomDownloadPath(cfg.YumRepo())
	calicoCtlUrl := fmt.Sprintf("url=%s/calicoctl-%s-linux-{.arch}", baseUrl, version)
	cfgString := "apiVersion: projectcalico.org/v3\nkind: CalicoAPIConfig\nmetadata:\nspec:\n  datastoreType: 'kubernetes'\n  kubeconfig: '/etc/kubernetes/admin.conf'"
	bkeNodes := bkenode.Nodes(bkeCluster.Spec.ClusterConfig.Nodes)

	downloadCommand := command.Custom{
		BaseCommand: command.BaseCommand{
			Ctx:             ctx,
			NameSpace:       bkeCluster.Namespace,
			Client:          c,
			Scheme:          scheme,
			OwnerObj:        bkeCluster,
			ClusterName:     bkeCluster.Name,
			Unique:          true,
			RemoveAfterWait: true,
		},
		Nodes:        bkeNodes.Master(),
		CommandName:  "config-download-calicoctl",
		CommandLabel: command.BKEClusterLabel,
	}

	commandSpec := command.GenerateDefaultCommandSpec()
	commandSpec.Commands = []agentv1beta1.ExecCommand{
		{
			ID: "download-calicoctl",
			Command: []string{
				downloadplugin.Name,
				calicoCtlUrl,
				"rename=calicoctl",
				"saveto=/usr/bin",
				"chmod=755",
			},
			Type:          agentv1beta1.CommandBuiltIn,
			BackoffIgnore: false,
		},
		{
			ID: "calicoctl-config-dir",
			Command: []string{
				"sudo mkdir -p /etc/calico",
			},
			Type:          agentv1beta1.CommandShell,
			BackoffIgnore: false,
		},
		{
			ID: "calicoctl-config",
			Command: []string{
				fmt.Sprintf("sudo echo -e %q > /etc/calico/calicoctl.cfg", cfgString),
			},
			Type:          agentv1beta1.CommandShell,
			BackoffIgnore: false,
		},
	}
	downloadCommand.CommandSpec = commandSpec

	if err := downloadCommand.New(); err != nil {
		return errors.Errorf("create download calicoctl command failed: %s", err.Error())
	}

	err, successNodes, failedNodes := downloadCommand.Wait()
	if err != nil {
		return errors.Errorf("wait download calicoctl command failed: %s", err.Error())
	}
	if len(failedNodes) > 0 {
		errInfo := "download-calicoctl command run failed"
		commandErrs, err := phaseutil.LogCommandFailed(*downloadCommand.Command, failedNodes, log, "download calicoctl failed")
		phaseutil.MarkNodeStatusByCommandErrs(bkeCluster, commandErrs)
		return errors.Errorf("%s, download calicoctl command run failed in flow nodes: %v, err: %v", errInfo, strings.Join(failedNodes, ","), err)
	}
	log.Info("BeforeCreateCustomOperate", "download calicoctl command run success in flow nodes: %v", strings.Join(successNodes, ","))
	return nil

}

func (e *EnsureAddonDeploy) saveAddonManifestsPostHook(_ phaseframe.Phase, _ error) error {
	ctx, c, bkeCluster, scheme, log := e.Ctx.Untie()

	if e.addonRecorders != nil && len(e.addonRecorders) != 0 {
		bkeNodes := bkenode.Nodes(bkeCluster.Spec.ClusterConfig.Nodes)
		for _, recorder := range e.addonRecorders {
			// convert recorder to agent command ,one recorder one command
			if len(recorder.AddonObjects) == 0 || recorder.AddonObjects == nil {
				continue
			}

			commandName := fmt.Sprintf("save-addon-manifests-%s-%s", strings.ToLower(recorder.AddonName), recorder.AddonVersion)
			saveAddonManifestsCommand := command.Custom{
				BaseCommand: command.BaseCommand{
					Ctx:             ctx,
					NameSpace:       bkeCluster.Namespace,
					Client:          c,
					Scheme:          scheme,
					OwnerObj:        bkeCluster,
					ClusterName:     bkeCluster.Name,
					Unique:          true,
					RemoveAfterWait: true,
				},
				Nodes:        bkeNodes.Master(),
				CommandName:  commandName,
				CommandLabel: command.BKEClusterLabel,
			}

			commandSpec := command.GenerateDefaultCommandSpec()

			addonManifestsDir := fmt.Sprintf("%s/%s-%s", constant.AddonManifestsDir, recorder.AddonName, recorder.AddonVersion)
			commands := []agentv1beta1.ExecCommand{}

			for _, obj := range recorder.AddonObjects {
				cmd := ""
				objFileName := ""
				if obj.NameSpace != "" {
					objFileName = fmt.Sprintf("%s/%s_%s_%s.yaml", addonManifestsDir, obj.Kind, obj.NameSpace, obj.Name)
					cmd = fmt.Sprintf("KUBECONFIG=/etc/kubernetes/admin.conf kubectl get %s -n %s %s -oyaml > %s", obj.Kind, obj.NameSpace, obj.Name, objFileName)
				} else {
					objFileName = fmt.Sprintf("%s/%s_%s.yaml", addonManifestsDir, obj.Kind, obj.Name)
					cmd = fmt.Sprintf("KUBECONFIG=/etc/kubernetes/admin.conf kubectl get %s %s -oyaml > %s", obj.Kind, obj.Name, objFileName)
				}
				command := agentv1beta1.ExecCommand{
					ID:            objFileName,
					Command:       []string{cmd},
					Type:          agentv1beta1.CommandShell,
					BackoffIgnore: false,
				}

				commands = append(commands, command)
			}

			commandSpec.Commands = []agentv1beta1.ExecCommand{
				{
					ID: "mkdir",
					Command: []string{
						fmt.Sprintf("mkdir -p %s", addonManifestsDir),
					},
					Type:          agentv1beta1.CommandShell,
					BackoffIgnore: false,
					BackoffDelay:  3,
				},
			}
			commandSpec.Commands = append(commandSpec.Commands, commands...)
			saveAddonManifestsCommand.CommandSpec = commandSpec

			if err := saveAddonManifestsCommand.New(); err != nil {
				log.Warn(constant.AddonDeployedReason, "failed to create save addon manifests command: %v", err)
				continue
			}
			// Don't wait ttl 10m will delete this command
		}
	}

	return nil
}
